{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f7e54da5-97a7-4447-ba2b-0ad24dd3de20",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "from glob import glob\n",
    "from unidiff import PatchSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6383506c-3405-4344-bfdd-6008c30a8e26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'devin-swebench-results'...\n",
      "remote: Enumerating objects: 582, done.\u001b[K\n",
      "remote: Counting objects: 100% (582/582), done.\u001b[K\n",
      "remote: Compressing objects: 100% (570/570), done.\u001b[K\n",
      "remote: Total 582 (delta 12), reused 579 (delta 9), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (582/582), 571.31 KiB | 6.35 MiB/s, done.\n",
      "Resolving deltas: 100% (12/12), done.\n",
      "Updating files: 100% (580/580), done.\n"
     ]
    }
   ],
   "source": [
    "!git clone git@github.com:CognitionAI/devin-swebench-results.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0afd1c6b-88e7-4e18-b065-f035f85c34b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_devin_txt_to_pred(pred_file):\n",
    "    inst_id = pred_file.split(\"/\")[-1].split(\"-diff\")[0]\n",
    "    pred = open(pred_file).read()\n",
    "    try:\n",
    "        PatchSet(pred)\n",
    "    except:\n",
    "        print(f\"{inst_id}: Prediction patch is malformed\")\n",
    "    return {\n",
    "        \"model_name_or_path\": \"devin-20240406\",\n",
    "        \"instance_id\": inst_id,\n",
    "        \"model_patch\": pred\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f81ead15-bc56-4cc7-ba0a-de2b68473c2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "570"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = []\n",
    "for pred_file in \\\n",
    "    glob(\"devin-swebench-results/output_diffs/fail/*.txt\") + \\\n",
    "    glob(\"devin-swebench-results/output_diffs/pass/*.txt\"):\n",
    "    predictions.append(convert_devin_txt_to_pred(pred_file))\n",
    "len(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cf22d4d5-5ba7-4b7a-a298-676f1955da0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model_name_or_path': 'devin-20240406',\n",
       " 'instance_id': 'django__django-16745',\n",
       " 'model_patch': 'diff --git a/django/core/validators.py b/django/core/validators.py\\nindex 6c622f5788..7a1aff3fe5 100644\\n--- a/django/core/validators.py\\n+++ b/django/core/validators.py\\n@@ -397,8 +397,9 @@ class StepValueValidator(BaseValidator):\\n     message = _(\"Ensure this value is a multiple of step size %(limit_value)s.\")\\n     code = \"step_size\"\\n \\n-    def compare(self, a, b):\\n-        return not math.isclose(math.remainder(a, b), 0, abs_tol=1e-9)\\n+    def compare(self, a, b, min_value=0):\\n+        offset = a - min_value\\n+        return not math.isclose(math.remainder(offset, b), 0, abs_tol=1e-9)\\n \\n \\n @deconstructible\\n'}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "00c2e805-cf64-4975-bd23-0b5d2be8576d",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"devin_predictions.jsonl\", \"w\") as f:\n",
    "    for pred in predictions:\n",
    "        print(json.dumps(pred), file=f, flush=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
